1. Calibrate the Camera

Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.

In [1]:
# import modules for this project
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
In [2]:
import glob

# Read in and make a list of calibration chessboard images 
chessboard_images = glob.glob('./camera_cal/calibration*.jpg') # Chessboard images
test_images = glob.glob('./test_images/*.jpg')  # Test images
In [3]:
# Arrays to store object points and image points from all the images
objpoints = [] # 3D points in real world space
imgpoints = [] # 2D points in image plane

# Prepare object points, like (0, 0, 0), (1, 0, 0) ..., (8, 5, 0)
objp = np.zeros((6*9, 3), np.float32)
# Generate the x and y coordinates and shape them into two columns
objp[:,:2] = np.mgrid[0:9,0:6].T.reshape(-1,2) 

for fname in chessboard_images:
    # read in each image
    img = mpimg.imread(fname)
    
    # Convert image to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

    # Find the chessboard corners
    ret, corners = cv2.findChessboardCorners(gray, (9,6), None)

    # If coners are found, add object points, image points
    if ret == True:
        
        imgpoints.append(corners)
        objpoints.append(objp)

        # draw detected corners on the chessboard image and store it 
        name = './output_images/draw_corners/' + fname.split('/')[-1]
        img_draw = cv2.drawChessboardCorners(img, (9,6), corners, ret)
        plt.imshow(img_draw)
        plt.savefig(name)
        
cv2.destroyAllWindows()
In [4]:
# Save the objpoints and imgpoints arrays
np.save('./output_images/imgpoints.npy', imgpoints)
np.save('./output_images/objpoints.npy', objpoints)

2. Correct for Image Distortion

Apply a distortion correction to raw images.

In [5]:
def cal_undistort(img):
    """
    The function performs the camera calibration
    and images distortion correction, 
    returns the undistorted image. 
    [img] a distorted 2D image
    [objpoints] the coordinates of the corners in undistorted 3D images
    [imgpoints] the coordinates of the corners in distorted 2D images.
    """
    # Load objpoints and imgpoints
    imgpoints = np.load('./output_images/imgpoints.npy')
    objpoints = np.load('./output_images/objpoints.npy')
    # Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Find the camera matrix and distortion coefficients to transform 3D image points to 2D image points
    ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
    # Undistort the input image
    undist = cv2.undistort(img, mtx, dist, None, mtx)
    
    return undist    
In [6]:
img_chessboard = mpimg.imread('./camera_cal/calibration3.jpg')
img_lane = mpimg.imread(test_images[0])
In [7]:
undistorted1 = cal_undistort(img_chessboard)
undistorted2 = cal_undistort(img_lane)
In [8]:
f, arr = plt.subplots(2, 2, figsize=(8, 6))
f.tight_layout()
arr[0,0].set_title('Original Image', fontsize=20)
arr[0,0].imshow(img_chessboard)
arr[1,0].set_title('Original Image', fontsize=20)
arr[1,0].imshow(img_lane)
arr[0,1].set_title('undistorted Image', fontsize=20)
arr[0,1].imshow(undistorted1)
arr[1,1].set_title('undistorted Image', fontsize=20)
arr[1,1].imshow(undistorted2)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
plt.savefig('./output_images/undistorted_test.jpg')

3. Implement a Color & Gradient Threshold

Use color transform, gradients, etc., to create a threshold binary image.

Note: Make sure you use the correct grayscale conversion depending on how you've read in your images. Use cv2.COLOR_RGB2GRAY if you've read in an image using mpimg.imread(). Use cv2.COLOR_BGR2GRAY if you've read in an image using cv2.imread().

3.1 Load and display original test images

Those test images are used to:

  • check my threshold functions
  • tune and get the best parameter values for my threshold functions

3.1.1 Import snapshots of project_video.mp4 as test images

Import four snapshots of the project_video.mp4 which failed in my 1st submission as additional test images.

In [9]:
vidcap = cv2.VideoCapture('./project_video.mp4')
times = [23000, 28000, 23750, 42000]
i = 0
for t in times:
    vidcap.set(cv2.CAP_PROP_POS_MSEC, t)  # just cue to t/1000 sec. position
    success, image = vidcap.read()
    if success:
        i+=1
        name = './test_images/snapshot' + str(i) + '.jpg'
        cv2.imwrite(name,image)

3.1.2 Load all test images

In [10]:
# load all test images
images= []
names = []
for fname in test_images:
    # load and append test image
    img = mpimg.imread(fname)
    images.append(img)
    # pick image name out of file path
    name = fname.split('/')[-1].split('.')[0]
    names.append(name)
In [11]:
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    r = i//4  # row of image
    c = i - r*4  # colomn of image
    arr[r,c].imshow(images[i], cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/load_images.jpg')

3.2 Color Thresholds

3.2.1 HLS threshold

As to color space, HLS can be more robust than RGB. Here I'll read in the same original image (straight lines1), convert to sparated H,L and S channels to get the following results:

In [12]:
hls_0 = cv2.cvtColor(img_lane, cv2.COLOR_RGB2HLS)
H_0 = hls_0[:,:,0]
L_0 = hls_0[:,:,1]
S_0 = hls_0[:,:,2]

f, arr = plt.subplots(1, 3, figsize=(24, 9))
f.tight_layout()
arr[0].set_title('H_channel', fontsize=30)
arr[0].imshow(H_0, cmap='gray')
arr[1].set_title('L_channel', fontsize=30)
arr[1].imshow(L_0, cmap='gray')
arr[2].set_title('S_channel', fontsize=30)
arr[2].imshow(S_0, cmap='gray')

plt.savefig('./output_images/thresholds/HLS_compare.jpg')

The S channel picks up the lines well, so I'll apply a threshold on it.

In [13]:
def hls_select(img, thresh=(0, 255)):
    """
    The fucntion applies threshold on the S-channel of HLS.
    """
    # Select the S channel as it picks up the lines well
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    s_channel = hls[:,:,2] 
    # Apply a threshold on S channel
    binary_output = np.zeros_like(s_channel)
    binary_output[(s_channel > thresh[0]) & (s_channel <= thresh[1])] = 1
    
    return binary_output

I tune the parameter as hls_select(img, thresh=(80, 255))

In [14]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    binary_output = hls_select(images[i], thresh=(80, 255))
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(binary_output, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/HLS.jpg')

3.2.2 RGB threshold

The HLS threshold works pretty good, but there're still some shadow areas in white after using hls_select(), such as frame42 and test4. Those white shadow area will be noise in the following lane line detection.

I'll take use rgb threshold to convert those shadow into black. Thus the RGB threshold should keep lane line in white and shadows which not detected by HLS threshold in black, no matter whether other areas are in white or black.

In [15]:
rgb_0 = img_lane
R_0 = rgb_0[:,:,0]
G_0 = rgb_0[:,:,1]
B_0 = rgb_0[:,:,2]

f, arr = plt.subplots(1, 3, figsize=(24, 9))
f.tight_layout()
arr[0].set_title('R_channel', fontsize=30)
arr[0].imshow(R_0, cmap='gray')
arr[1].set_title('G_channel', fontsize=30)
arr[1].imshow(G_0, cmap='gray')
arr[2].set_title('B_channel', fontsize=30)
arr[2].imshow(B_0, cmap='gray')

plt.savefig('./output_images/thresholds/HLS_compare.jpg')

The R channel picks up the lines well, so I'll apply a threshold on it.

In [16]:
def rgb_select(img, thresh=(0, 255)):
    """
    The fucntion applies threshold on the R-channel of RGB.
    """
    # Select the R channel as it picks up the lines well
    r_channel = img[:,:,0] 
    # Apply a threshold on S channel
    binary_output = np.zeros_like(r_channel)
    binary_output[(r_channel > thresh[0]) & (r_channel <= thresh[1])] = 1
    
    return binary_output

I tune the parameter as rgb_select(img, thresh=(50, 255))

In [17]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    binary_output = rgb_select(images[i], thresh=(50, 255))
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(binary_output, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/RGB.jpg')

3.2.3 Combine HLS and RGB threshold

To summarize, I tune all color threshold parameters as below:

  • hls_select(img, thresh=(80, 255))
  • rgb_select(img, thresh=(50, 255))
In [18]:
def combine_color(img):
    """
    The function combines S_channel and R_channel to set color thresholds.
    """
    # Apply each of the thresholding functions
    s_binary = hls_select(img, thresh=(80, 255))
    r_binary = rgb_select(img, thresh=(50, 255))
    combined = np.zeros_like(s_binary)
    combined[(s_binary==1) & (r_binary==1)] = 1
    
    return combined
In [19]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    binary_output = combine_color(images[i])
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(binary_output, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/combine_color.jpg')

3.3 Gradient Threshold

3.3.1 Absolute Value of the Gradient

In [20]:
def abs_sobel_thresh(img, orient='x', thresh=(0,255)):
    """
    The function applies Sobel x and y,
    then takes an absolute value and applies a threshold.
    """
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Calculate directional gradient
    if orient == 'x':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0))
    elif orient == 'y':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1))
    # Rescale back to 8 bit integer
    scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
    # Apply threshold
    binary_output = np.zeros_like(scaled_sobel)
    binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1
    
    return binary_output

To apply threshold on the absolute value of the gradient in x direction, I tune the parameter as abs_sobel_thresh(img, orient='x', thresh=(20, 200))

In [21]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    binary_output_i = abs_sobel_thresh(images[i], orient='x', thresh=(20, 200))
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(binary_output_i, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/abs_sobelx_thresh.jpg')

To apply threshold on the absolute value of the gradient in y direction, I tune the parameter as abs_sobel_thresh(img, orient='y', thresh=(20, 200))

In [22]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    binary_output_i = abs_sobel_thresh(images[i], orient='y', thresh=(20, 200))
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(binary_output_i, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/abs_sobely_thresh.jpg')

3.3.2 Magnitude of the Gradient

In [23]:
def mag_thresh(img, sobel_kernel, thresh=(0, 255)):
    """
    The function returns the magnitude of the gradient
    for a given sobel kernel size and threshold values.
    """
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Calculate gardient magnitude
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    gradmag = np.sqrt(sobelx**2, sobely**2)
    # Rescale to 8 bit
    scale_factor = np.max(gradmag)/255
    gradmag = (gradmag/scale_factor).astype(np.uint8)   
    # Apply threshold
    binary_output = np.zeros_like(gradmag)
    binary_output[(gradmag >= thresh[0]) & (gradmag <= thresh[1])] = 1
    
    return binary_output

I tune the parameter as mag_thresh(img, sobel_kernel=3, thresh=(30, 100))

In [24]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    binary_output = mag_thresh(images[i], sobel_kernel=3, thresh=(30, 100))
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(binary_output, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/mag_thresh.jpg')

3.3.3 Direction of the Gradient

In [25]:
def dir_threshold(img, sobel_kernel, thresh=(0, np.pi/2)):
    """
    The function applies Sobel x and y, then compute the
    direction of the gradient and applies a threshold.
    """
    #gray = hls_select(img, thresh=(90, 255))
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Calculate gradient direction
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    absgraddir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
    # Apply threshold
    binary_output = np.zeros_like(absgraddir)
    binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1
    
    return binary_output

I tune the parameter as dir_threshold(img, sobel_kernel=9, thresh=(0.7, 1.2))

In [26]:
# Test function and tune parameters to get ideal output
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    binary_output = dir_threshold(images[i], sobel_kernel=9, thresh=(0.7, 1.2))
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(binary_output, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/dir_threshold.jpg')

3.3.4 Combing Gradient Thresholds

To summarize, I tune all gradient threshold parameters as below:

  • abs_sobel_thresh(img, orient='x', thresh=(20, 200))
  • abs_sobel_thresh(img, orient='x', thresh=(20, 200))
  • mag_thresh(img, sobel_kernel=3, thresh=(30, 100))
  • dir_threshold(img, sobel_kernel=9, thresh=(0.7, 1.2))
In [27]:
def combine_gradient(img):
    """
    The function combines gradient, magnitude of gradient and 
    direction of gradient thresholds.
    """
    # Apply each of the thresholding functions
    gradx = abs_sobel_thresh(img, orient='x', thresh=(20, 200))
    grady = abs_sobel_thresh(img, orient='y', thresh=(20, 200))
    mag_binary =  mag_thresh(img, sobel_kernel=3, thresh=(30, 100))
    dir_binary = dir_threshold(img, sobel_kernel=9, thresh=(0.7, 1.2))

    combined = np.zeros_like(dir_binary)
    combined[((gradx==1) & (grady==1)) | ((mag_binary==1) & (dir_binary==1))] = 1
    
    return combined
In [28]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    binary_output = combine_gradient(images[i])
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(binary_output, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/combine_gradient.jpg')

3.4 Combining Color and Gradient Thresholds

In [29]:
def color_grad(img):
    """
    The function combines HLS color threshold and multiple gradient thresholds.
    """
    color_binary = combine_color(img)
    gradient_binary = combine_gradient(img)
    combined_binary = np.zeros_like(color_binary)
    combined_binary[(color_binary == 1) | (gradient_binary == 1)] = 1
    
    return combined_binary
In [30]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    binary_output = color_grad(images[i])
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(binary_output, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/thresholds/color_grad.jpg')
In [31]:
# Run the function
combine_binary = color_grad(img_lane)
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(img_lane)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(combine_binary, cmap='gray')
ax2.set_title('Color & Gradient', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
plt.savefig('./output_images/thresholds/thresholds.jpg')

4. Warp the Image Using Perspective Transform

Apply a perspective transform to rectify binary image("birds-eye view").

4.1 Choose Source and Destination Points

In [32]:
img_size = (1280, 720) 

src = np.int32(
    [[(img_size[0] * 5 / 6) + 60, img_size[1]],
    [(img_size[0] / 2 + 65), img_size[1] / 2 + 100],
    [(img_size[0] / 2) - 60, img_size[1] / 2 + 100],
    [((img_size[0] / 6) - 10), img_size[1]]])
dst = np.int32(
    [[(img_size[0] * 5 / 6), img_size[1]],
    [(img_size[0] * 5 / 6), 0],
    [(img_size[0] / 6), 0],
    [(img_size[0] / 6), img_size[1]]])
In [33]:
src
Out[33]:
array([[1126,  720],
       [ 705,  460],
       [ 580,  460],
       [ 203,  720]], dtype=int32)
In [34]:
dst
Out[34]:
array([[1066,  720],
       [1066,    0],
       [ 213,    0],
       [ 213,  720]], dtype=int32)

Apply the source (original) coordinates src and destination (desired or warped) coordinates dst above in the following warper function.

In [35]:
def warper(img):
    """
    Define caliation box in source (original)
    and destination (desired or warped) coordinates. 
    """
    img_size = (img.shape[1], img.shape[0])
    # Four source coordinates
    src = np.float32([[1126,  720],
                      [ 705,  460],
                      [ 580,  460],
                      [ 203,  720]])
    # Four desired coordinates
    dst = np.float32([[1066, 720],
                      [1066,   0],
                      [ 213,   0],
                      [ 213, 720]])
    # Compute the perspective transform M
    M = cv2.getPerspectiveTransform(src, dst)
    # Compute the inverse perspective transform Minv
    Minv = cv2.getPerspectiveTransform(dst, src)
    # Create warped image - uses linear interpolation
    warped = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)
    
    return warped, M, Minv

4.2 Verify Source and Destination Points

I verified that my perspective transform was working as expected by drawing the src and dst points onto two test images with straight lines and their warped counterparts respectively to verify that the lines appear parallel in their corresponding warped image.

In [36]:
# Read test images with straight lines
straight_lines1 = mpimg.imread('./test_images/straight_lines1.jpg')
straight_lines2 = mpimg.imread('./test_images/straight_lines2.jpg')

# Undistort the images
undist_sl1 = cal_undistort(straight_lines1)
undist_sl2 = cal_undistort(straight_lines2)

# Warp the undistorted images
warp_sl1, _, _ = warper(undist_sl1)
warp_sl2, _, _ = warper(undist_sl2)
In [37]:
# Run the function
combine_binary = color_grad(img_lane)
# Plot the result
f, arr = plt.subplots(2, 2, figsize=(10, 8))
f.tight_layout()

arr[0,0].imshow(undist_sl1, extent=(0,1280,720,0))
arr[0,0].plot([1126,  705,  580,  203],
              [ 720,  460,  460,  720],'r-',linewidth=2)
arr[0,0].set_title('Udistorted Image_1 with [src] drawn', fontsize=18)

arr[0,1].imshow(undist_sl2, extent=(0,1280,720,0))
arr[0,1].plot([1126,  705,  580,  203],
              [ 720,  460,  460,  720],'r-',linewidth=2)
arr[0,1].set_title('Udistorted Image_2 with [src] drawn', fontsize=18)

arr[1,0].imshow(warp_sl1, cmap='gray', extent=(0,1280,720,0))
arr[1,0].plot([213, 213], [  0, 720],'r-',linewidth=2)
arr[1,0].plot([1066,1066], [  0, 720],'r-',linewidth=2)
arr[1,0].set_title('Warped result_1 with [dst] drawn', fontsize=18)

arr[1,1].imshow(warp_sl2, cmap='gray', extent=(0,1280,720,0))
arr[1,1].plot([213, 213], [  0, 720],'r-',linewidth=2)
arr[1,1].plot([1066,1066], [  0, 720],'r-',linewidth=2)
arr[1,1].set_title('Warped result_2 with [dst] drawn', fontsize=18)

plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
plt.savefig('./output_images/src&dst_drawn.jpg')

4.3 Warp a Binary Image

I pick up all undistortion, threshold and warper function together to get a function warp_camera(img_camera) which take a original distorted image as input and output undistorted and warped binary image in bird-eye perspection.

In [38]:
def warp_camera(img_camera):
    """
    The input is the original image taken by camera.
    The function undistort the original image, apply a color&gradient threshold
    on the undistorted image and warp the binary image in bird-eye perspection.
    """
    img_undistort = cal_undistort(img_camera)
    img_threshold = color_grad(img_undistort)
    img_warped, M, Minv = warper(img_threshold)
    return img_undistort, img_warped, M, Minv
In [39]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    _, warped_i, _, _ = warp_camera(images[i])
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(warped_i, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/warped_binary_image.jpg')

5. Decide Which Pixels Are Lane Line Pixels

Detect lane pixels and fit to find the lane boundary

5.1 Window Fitting

Find window from the bottom to the top of the warped image.

In [40]:
def window_mask(width, height, img_ref, center,level):
    """
    The function is used to draw window areas.
    """
    output = np.zeros_like(img_ref)
    output[int(img_ref.shape[0]-(level+1)*height):int(img_ref.shape[0]-level*height),max(0,int(center-width/2)):min(int(center+width/2),img_ref.shape[1])] = 1
    return output

5.1.1 Unidirectional Window Fitting

In [41]:
def find_window_centroids(binary_warped, window_width, window_height, margin):
    """
    The function finds all the left and right window centroids 
    for each level in the given binary image.
    """
    window_centroids = [] # Store the (left, right) window centroid positions per level
    window = np.ones(window_width) # Create out window template that we will use for convolutions
    
    # Sum quarter bottom of image to get slice, could use a different ratio
    l_sum = np.sum(binary_warped[int(3*binary_warped.shape[0]/4):,:int(binary_warped.shape[1]/2)], axis=0)
    l_center = np.argmax(np.convolve(window,l_sum))-window_width/2
    
    r_sum = np.sum(binary_warped[int(3*binary_warped.shape[0]/4):,int(binary_warped.shape[1]/2):], axis=0)
    r_center = np.argmax(np.convolve(window,r_sum))-window_width/2+int(binary_warped.shape[1]/2)

    
    # Add what we found for the first layer
    window_centroids.append((l_center,r_center))
    
    # Go through each layer looking for max pixel locations
    for level in range(1,int(binary_warped.shape[0]/window_height)):
        # Convolve the window into the vertical slice of the image
        layer_bottom = int(binary_warped.shape[0]-(level+1)*window_height)
        layer_top = int(binary_warped.shape[0]-level*window_height)
        image_layer = np.sum(binary_warped[layer_bottom:layer_top, :], axis=0)
        conv_signal = np.convolve(window, image_layer)
        # Use window_width/2 as offset because convolution signal reference is at right side of window, not center of window
        offset = window_width/2
        # Find the best left centroid by using past left center as a reference
        l_min_index = int(max(l_center+offset-margin,0))
        l_max_index = int(min(l_center+offset+margin,binary_warped.shape[1]))
        l_center = np.argmax(conv_signal[l_min_index:l_max_index])+l_min_index-offset
        # Find the best right centroid by using past right center as a reference
        r_min_index = int(max(r_center+offset-margin,0))
        r_max_index = int(min(r_center+offset+margin,binary_warped.shape[1]))
        r_center = np.argmax(conv_signal[r_min_index:r_max_index])+r_min_index-offset
        # Add what we found for that layer
        window_centroids.append((l_center,r_center))
        
    return window_centroids
In [42]:
def mark_centroids(binary_warped, window_centroids, window_width, window_height):
    """
    The function find and mark left and right centroids.
    """
    # If we found any window centers
    if len(window_centroids) > 0:

        # Points used to draw all the left and right windows
        l_points = np.zeros_like(binary_warped)
        r_points = np.zeros_like(binary_warped)

        # Go through each level and draw the windows
        for level in range(0,len(window_centroids)):
            # Window_mask is a function to draw window areas
            l_mask = window_mask(window_width,window_height,binary_warped,window_centroids[level][0],level)
            r_mask = window_mask(window_width,window_height,binary_warped,window_centroids[level][1],level)
            # Add graphic points from window mask here to total pixels found
            l_points[(l_points == 255) | ((l_mask == 1))] = 255
            r_points[(r_points == 255) | ((r_mask == 1))] = 255

        # Draw the results
        template = np.array(r_points+l_points,np.uint8) # add both left and right window pixels together
        zero_channel = np.zeros_like(template) # create a zero color channel
        template = np.array(cv2.merge((zero_channel,template,zero_channel)),np.uint8) # make window pixels green 
        warpage = np.dstack((binary_warped, binary_warped, binary_warped))*255 # making the original road pixels 3 color channels
        output = cv2.addWeighted(warpage, 1, template, 0.5, 0.0) # overlay the original road image with window results 

    # If no window centers found, just display original road image 
    else:
        output = np.array(cv2.merge((binary_warped, binary_warped, binary_warped)),np.uint8)
    
    return output
In [43]:
# Window settings
window_width = 50
window_height = 80 # Break image into 9 vertical layers since image height is 720
margin = 50 # How much to slide left and right for searching
In [44]:
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    _, warped_i, _, _ = warp_camera(images[i])
    window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
    mark_centroids_i = mark_centroids(warped_i, window_centroids_i, window_width, window_height)
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(mark_centroids_i, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/window_fitting_result.jpg')

5.1.2 Bidirectional Window Fitting

As the output images shown above, the find_window_centroids works well in the front half part of the levels but tends to be out of order in the back half part of the levels.

Thus I decide to apply the find_window_centroids in both directions (from bottom to top as well as from top to bottom) and choose the better left and right centroids for each level repectively.

np.flipud() is used twice to upside down the binary warped image as an input, as well as the find_window_centroids() output to make the orders of the level in the same direction

In [45]:
def better_window_centroids(binary_warped, window_width, window_height, margin):
    """
    The function find window centroids in bidirection (bottom to top & top to bottom)
    and then selects better left and right centroids for each level.
    """
    # Find window centroids in both directions
    window_centroids_up = find_window_centroids(binary_warped, window_width, window_height, margin)
    window_centroids_down = np.flipud(find_window_centroids(np.flipud(binary_warped), window_width, window_height, margin))
   
    window_centroids = []
    index = []
    for level in range(1,int(binary_warped.shape[0]/window_height)): 
        # Fitting Direction: Bottom -> Top
        # Identify window areas
        l_mask_up = window_mask(window_width,window_height,binary_warped,window_centroids_up[level][0],level)
        r_mask_up = window_mask(window_width,window_height,binary_warped,window_centroids_up[level][1],level)
        l_mask_up = np.array(l_mask_up)
        r_mask_up = np.array(r_mask_up)
        # Identify the nonzero pixels in x and y within the window
        good_left_up = (binary_warped*l_mask_up).nonzero()
        good_right_up = (binary_warped*r_mask_up).nonzero() 
        
        # Fitting Direction: Top -> Bottom
        # Identify window areas
        l_mask_down = window_mask(window_width,window_height,binary_warped,window_centroids_down[level][0],level)
        r_mask_down = window_mask(window_width,window_height,binary_warped,window_centroids_down[level][1],level)
        l_mask_down = np.array(l_mask_down)
        r_mask_down = np.array(r_mask_down)
        # Identify the nonzero pixels in x and y within the window
        good_left_down = (binary_warped*l_mask_down).nonzero()
        good_right_down = (binary_warped*r_mask_down).nonzero()
        
        # Which is better
        # for left
        l_center = window_centroids_up[level][0]
        if len(good_left_up[0]) < len(good_left_down[0]):
            l_center = window_centroids_down[level][0]
        # for right
        r_center = window_centroids_up[level][1]
        if len(good_right_up[0]) < len(good_right_down[0]):
            r_center = window_centroids_down[level][1]
        
        window_centroids.append((l_center, r_center))
    
    return window_centroids
In [46]:
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    _, warped_i, _, _ = warp_camera(images[i])
    window_centroids_i = better_window_centroids(warped_i, window_width, window_height, margin)
    mark_centroids_i = mark_centroids(warped_i, window_centroids_i, window_width, window_height)
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(mark_centroids_i, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/better_fitting_result.jpg')

5.2 Find and Plot the Track Line

Using the window centroids found by bidirectional window fitting method to plot the track line.

In [47]:
def find_ploty(binary_warped, window_centroids, window_height, window_width):
    """
    The function extract left and right line pixel positions
    to fit a second order polynomial to both, and generate x and y for plotting.
    """
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Set minimum number of pixels found to recenter window
    minpix = 100
    # Create empty lists to receive valid left and right lane point coordinates
    leftx = []
    lefty = []
    rightx = []
    righty = []

    # Go through each level to find
    for level in range(0, len(window_centroids)):

        # Identify window areas
        l_mask = window_mask(window_width,window_height,binary_warped,window_centroids[level][0],level)
        r_mask = window_mask(window_width,window_height,binary_warped,window_centroids[level][1],level)
        l_mask = np.array(l_mask)
        r_mask = np.array(r_mask)
        # Identify the nonzero pixels in x and y within the window
        good_left = (binary_warped*l_mask).nonzero()
        good_right = (binary_warped*r_mask).nonzero()
        # Append these coordinates to the lists
        lefty.append(good_left[0])
        leftx.append(good_left[1])
        righty.append(good_right[0])
        rightx.append(good_right[1])
    
    # Concatenate the arrays of nonzero pixel coordinates
    lefty = np.concatenate(lefty)
    leftx = np.concatenate(leftx)
    righty = np.concatenate(righty)
    rightx = np.concatenate(rightx)

    # Fit a second order polynomial to each
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)
    # Generate x and y values for plotting
    ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0])
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    
    # Pack all x and y coordinates to cut down the numbers of output
    lane = np.array([lefty, leftx, righty, rightx])
    fitx = np.array([left_fitx, right_fitx])
    
    return ploty, lane, fitx
In [48]:
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    _, warped_i, _, _ = warp_camera(images[i])
    window_centroids_i = better_window_centroids(warped_i, window_width, window_height, margin)
    ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    # Create an output image to draw on and visualize the result 
    out_img_i = np.dstack((warped_i, warped_i, warped_i))*255
    out_img_i[lane_i[0], lane_i[1]] = [255, 0, 0]
    out_img_i[lane_i[2], lane_i[3]] = [0, 0, 255]
    # Draw Lane Lines
    arr[r,c].imshow(out_img_i, cmap='gray', extent=(0,1280,720,0))
    arr[r,c].plot(fitx_i[0], ploty_i, color='yellow', linewidth=4)
    arr[r,c].plot(fitx_i[1], ploty_i, color='yellow', linewidth=4)
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/find_ploty.jpg')

6. Determine the Line Shape and Position

Determine the curvature of the lane and vehicle position with respect to center.

6.1 Measuring Curvature

In [49]:
def calculate_curve(A, B, y):
    """
    The function calculate the bottom radius of curvature of a second order
    polynomial curve f(y) = A*y**2 + B*y + C.
    """
    curve_rad = ((1 + (2*A*y + B)**2)**1.5) / np.absolute(2*A)
    
    return curve_rad
In [50]:
def measure_curve(ploty, lane):
    """
    The function calculates the radius of curcature after correcting
    for scale in x and y.
    """
    # Define y-value where we want radius of curvature
    # Here I'll choose the maximum y-value, corresponding to the bottom of the image
    y_eval = np.max(ploty)
    
    # Define conversions in x and y from pixels space to meters.
    ym_per_pix = 30/720 # meters per pixel in y dimension
    xm_per_pix = (3.7/700)*(3/4) # meters per pixel in x dimension
    
    # Fit a second order polynomial to each lane
    A_left, B_left, C_left = np.polyfit(lane[0]*ym_per_pix, lane[1]*xm_per_pix, 2)
    A_right, B_right, C_right = np.polyfit(lane[2]*ym_per_pix, lane[3]*xm_per_pix, 2)
    
    # Calculate the radius of each lane
    left_curverad = calculate_curve(A_left, B_left, y_eval*ym_per_pix)
    right_curverad = calculate_curve(A_right, B_right, y_eval*ym_per_pix)
    
    return left_curverad, right_curverad
In [51]:
# Test function
for i in range(len(images)):
    undistorted_i, warped_i, M_i, Minv_i = warp_camera(images[i])
    window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
    ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
    left_curverad_i, right_curverad_i = measure_curve(ploty_i, lane_i)
    print('['+ names[i] +']')
    print('R_left = {:0.2f}(m), R_right = {:0.2f}(m)'.format(left_curverad_i, right_curverad_i))
    
    # Take the average of the left and right curve rad as the lane curveture
    curvature_i = (left_curverad_i + right_curverad_i)/2
    print('Radius of Curvature = {:0.2f}(m)'.format(curvature_i))
    print()
[snapshot1]
R_left = 21186.25(m), R_right = 4318.20(m)
Radius of Curvature = 12752.22(m)

[snapshot2]
R_left = 854.74(m), R_right = 929.56(m)
Radius of Curvature = 892.15(m)

[snapshot3]
R_left = 15490.54(m), R_right = 1145.41(m)
Radius of Curvature = 8317.97(m)

[snapshot4]
R_left = 940.45(m), R_right = 951.98(m)
Radius of Curvature = 946.22(m)

[straight_lines1]
R_left = 2415.82(m), R_right = 3615.28(m)
Radius of Curvature = 3015.55(m)

[straight_lines2]
R_left = 2503.34(m), R_right = 9275.88(m)
Radius of Curvature = 5889.61(m)

[test1]
R_left = 5688.72(m), R_right = 1815.23(m)
Radius of Curvature = 3751.97(m)

[test2]
R_left = 726.83(m), R_right = 632.95(m)
Radius of Curvature = 679.89(m)

[test3]
R_left = 1395.48(m), R_right = 1013.39(m)
Radius of Curvature = 1204.44(m)

[test4]
R_left = 17898.69(m), R_right = 3441.74(m)
Radius of Curvature = 10670.22(m)

[test5]
R_left = 1049.39(m), R_right = 1062.38(m)
Radius of Curvature = 1055.89(m)

[test6]
R_left = 2182.19(m), R_right = 697.80(m)
Radius of Curvature = 1440.00(m)

6.2 Determine Vehicle Position

In [52]:
def car_position(fitx, x_median):
    xm_per_pix = (3.7/700)*(3/4) # meters per pixel in x dimension
    car_center = x_median * xm_per_pix  # take the image center in x direction as the center of the vehicle
    
    # Take the bottom x positions of the left and right lanes
    x_left = fitx[0][-1] * xm_per_pix
    x_right = fitx[1][-1] * xm_per_pix
    # Take the average of the x_left and x_right as the center of the lane
    lane_center = (x_left + x_right)/2
    
    if car_center+0.005 < lane_center:
        T_position = 'Vehicle is {:0.2f}m left of center'.format(lane_center-car_center)
        return T_position
    elif car_center-0.005 > lane_center:
        T_position = 'Vehicle is {:0.2f}m right of center'.format(car_center-lane_center)
        return T_position
    else:
        return 'Vehicl is on the center'
In [53]:
x_median = img_lane.shape[1]/2
x_median
Out[53]:
640.0
In [54]:
for i in range(len(images)):
    undistorted_i, warped_i, M_i, Minv_i = warp_camera(images[i])
    window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
    ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
    T_position_i = car_position(fitx_i, x_median)
    
    print('['+ names[i] +']')
    print(T_position_i)
[snapshot1]
Vehicle is 0.15m left of center
[snapshot2]
Vehicle is 0.15m left of center
[snapshot3]
Vehicle is 0.07m left of center
[snapshot4]
Vehicle is 0.08m right of center
[straight_lines1]
Vehicle is 0.03m right of center
[straight_lines2]
Vehicle is 0.01m right of center
[test1]
Vehicle is 0.12m left of center
[test2]
Vehicle is 0.20m left of center
[test3]
Vehicle is 0.08m left of center
[test4]
Vehicle is 0.24m left of center
[test5]
Vehicle is 0.04m right of center
[test6]
Vehicle is 0.22m left of center

6.3 Drawing Lane

In [55]:
def drawing(undistorted_image, binary_warped, ploty, fitx, Minv):
    """
    The funtion draw the area between detected lane lines on the undistorted original image.
    """
    # Create an image to draw the lines on
    warp_zero = np.zeros_like(binary_warped).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
    
    # Recast the x and y points into usable format for cv2.fillPoly()
    pts_left = np.array([np.transpose(np.vstack([fitx[0], ploty]))]) 
    pts_right = np.array([np.flipud(np.transpose(np.vstack([fitx[1], ploty])))])
    pts = np.hstack((pts_left, pts_right))
    
    # Draw the lane onto the warped blank image
    cv2.fillPoly(color_warp, np.int_([pts]), (0,255,0))
    
    # Warp the blank back to original image space using inversed perspective matrix (Minv)
    newwarp = cv2.warpPerspective(color_warp, Minv, (undistorted_image.shape[1], undistorted_image.shape[0]))
    # Combine the result with the original image
    result = cv2.addWeighted(undistorted_image, 1, newwarp, 0.3, 0)
    
    return result
In [56]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    undistorted_i, warped_i, M_i, Minv_i = warp_camera(images[i])
    window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
    ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
    result_i = drawing(undistorted_i, warped_i, ploty_i, fitx_i, Minv_i)
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(result_i, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/draw_lane_area.jpg')
In [57]:
def drawing_lane(undistorted_image, binary_warped, lane, Minv):
    """
    The funtion draw the detected left and right lane lines on the undistorted original image
    in red and blue respectively.
    """
    # Create an image to draw the lines on
    warp_zero = np.zeros_like(binary_warped).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
    
    # draw red left and blue right lane lines
    color_warp[lane[0], lane[1]] = [0, 255, 255]
    color_warp[lane[2], lane[3]] = [255, 255, 0]
    
    # Warp the blank back to original image space using inversed perspective matrix (Minv)
    newwarp = cv2.warpPerspective(color_warp, Minv, (undistorted_image.shape[1], undistorted_image.shape[0]))
    # Combine the result with the original image
    result = cv2.addWeighted(undistorted_image, 1, newwarp, -0.7, 0)
    
    return result
In [58]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    undistorted_i, warped_i, M_i, Minv_i = warp_camera(images[i])
    window_centroids_i = find_window_centroids(warped_i, window_width, window_height, margin)
    ploty_i, lane_i, fitx_i = find_ploty(warped_i, window_centroids_i, window_height, window_width)
    result_i = drawing_lane(undistorted_i, warped_i, lane_i, Minv_i)
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(result_i, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/draw_lane_line.jpg')

7. Output

Warp the detected lane boundaries back onto the original image.

Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.

7.1 Define Function to Process Image

In [59]:
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
In [60]:
def process_image(img_camera):
    '''
    The function:
    1. undistorts the original camera image,
    2. converts the undistorted image into a binary form via 
    applying color & gradient thresholds,
    3. warps the binary image into birds-eye perspective,
    4. detects lane curvature and position via the warped binary image,
    5. displays the curvature and position on the undistorted image,
    6. draws/marks the detected lane on the undistorted image.
    '''
    
    # Undistort the original image to draw lane lines on it later
    # Apply color&gradient thresholds and birds-eye perspective on the undistorted image
    undistorted_image, binary_warped, M, Minv = warp_camera(img_camera)
    # Detect the left and right lane lines
    window_centroids = find_window_centroids(binary_warped, window_width=50, window_height=80, margin=40)
    ploty, lane, fitx = find_ploty(binary_warped, window_centroids, window_height=80, window_width=50)
    
    # Detect lane curvature and position
    left_curverad, right_curverad = measure_curve(ploty, lane)
    curvature = (left_curverad + right_curverad)/2
    
    # Display lane curvature and position on the undistorted image
    font = cv2.FONT_HERSHEY_SIMPLEX
    T_rad = 'Radius of Curvature = {:0.2f}(m)'.format(curvature)
    T_position = car_position(fitx, 640)

    cv2.putText(undistorted_image, T_rad,(60,100), font, 1.5,(255,255,255), 3, cv2.LINE_AA)
    cv2.putText(undistorted_image, T_position,(60,150), font, 1.5,(255,255,255), 3, cv2.LINE_AA)
    
    # Drawing
    result = drawing_lane(undistorted_image, binary_warped, lane, Minv)
    result = drawing(result, binary_warped, ploty, fitx, Minv)
    
    return result
In [61]:
# Test function
f, arr = plt.subplots(3,4, figsize=(24, 12))
f.tight_layout()
for i in range(len(images)):
    result_i = process_image(images[i])
    r = i//4  # row of image 
    c = i - r*4  # colomn of image
    arr[r,c].imshow(result_i, cmap='gray')
    arr[r,c].set_title(names[i], fontsize=30)
plt.subplots_adjust(hspace = 0.3)
plt.savefig('./output_images/process_image.jpg')

7.2 Project Video

In [62]:
white_output1 = './test_videos_output/project_video.mp4'

# You may also uncomment the following line for a subclip of the first 5 seconds
#clip1 = VideoFileClip("./project_video.mp4").subclip(0,5)
clip1 = VideoFileClip("./project_video.mp4")
white_clip1 = clip1.fl_image(process_image)
%time white_clip1.write_videofile(white_output1, audio=False)
[MoviePy] >>>> Building video ./test_videos_output/project_video.mp4
[MoviePy] Writing video ./test_videos_output/project_video.mp4
100%|█████████▉| 1260/1261 [26:30<00:01,  1.27s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: ./test_videos_output/project_video.mp4 

CPU times: user 27min 8s, sys: 1min 30s, total: 28min 39s
Wall time: 26min 32s
In [63]:
HTML("""
<video width="1280" height="720" controls>
  <source src="{0}">
</video>
""".format(white_output1))
Out[63]:

7.3 Challenge Video

In [64]:
white_output2 = './test_videos_output/challenge_video.mp4'

# You may also uncomment the following line for a subclip of the first 5 seconds
clip2 = VideoFileClip("./challenge_video.mp4").subclip(0,5)
#clip2 = VideoFileClip("./challenge_video.mp4")
white_clip2 = clip2.fl_image(process_image)
%time white_clip2.write_videofile(white_output2, audio=False)
[MoviePy] >>>> Building video ./test_videos_output/challenge_video.mp4
[MoviePy] Writing video ./test_videos_output/challenge_video.mp4
100%|██████████| 150/150 [03:04<00:00,  1.25s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: ./test_videos_output/challenge_video.mp4 

CPU times: user 3min 10s, sys: 9.76 s, total: 3min 20s
Wall time: 3min 6s
In [65]:
HTML("""
<video width="1280" height="720" controls>
  <source src="{0}">
</video>
""".format(white_output2))
Out[65]:

7.3 Harder Challenge Video

In [66]:
white_output3 = './test_videos_output/harder_challenge_video.mp4'

# You may also uncomment the following line for a subclip of the first 5 seconds
clip3 = VideoFileClip("./harder_challenge_video.mp4").subclip(0,5)
#clip3 = VideoFileClip("./harder_challenge_video.mp4")
white_clip3 = clip3.fl_image(process_image)
%time white_clip3.write_videofile(white_output3, audio=False)
[MoviePy] >>>> Building video ./test_videos_output/harder_challenge_video.mp4
[MoviePy] Writing video ./test_videos_output/harder_challenge_video.mp4
 99%|█████████▉| 125/126 [02:39<00:01,  1.29s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: ./test_videos_output/harder_challenge_video.mp4 

CPU times: user 2min 44s, sys: 8.84 s, total: 2min 53s
Wall time: 2min 42s
In [67]:
HTML("""
<video width="1280" height="720" controls>
  <source src="{0}">
</video>
""".format(white_output3))
Out[67]: